/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package com.gr.search;

/**
 *
 * @author Hieudt
 */
import com.gr.sample.Indexer;
import com.gr.sample.Searcher;
import com.gr.context.QueryLog;
import com.gr.context.Profile;
import com.gr.analys.VietAnalyzer;
import com.gr.context.ContextManager;
import com.gr.zoie.DocIDMapper;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.event.HyperlinkEvent;
import org.apache.lucene.document.Document;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.StringReader;
import java.net.HttpURLConnection;
import javax.swing.event.HyperlinkListener;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.highlight.Fragmenter;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
import org.apache.lucene.search.highlight.QueryScorer;
import com.maxmind.geoip.*;
import com.mysql.jdbc.Connection;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import org.apache.lucene.queryParser.QueryParser;
import vn.hus.nlp.tokenizer.tokens.TaggedWord;

public class ContextSearcher {

    private final static float k_term = 1;
    private final static float k_inte = 10;
    private final static float k_loc = 1;
    private final static int K = 500;
    private final static int RESULT_LIMIT = 50;
    private final static float E = (float) 0.001;
    private static String IPstr;
    private QueryLog ql;
    private Profile user;
    private VietAnalyzer viA = new VietAnalyzer();
    private Mapper mapper = new Mapper();
    private DocIDMapper docidMapper;
    QueryParser parser = new QueryParser(Version.LUCENE_30, "contents", viA);
    QueryParser parser2 = new QueryParser(Version.LUCENE_30, "contents", new StandardAnalyzer(Version.LUCENE_30));
    ContextManager context;
    IndexSearcher is;
    Highlighter highlighter;
    Location l;

    public ContextSearcher(int id, String ip) {
        try {
            context = new ContextManager(id, ip);
            //Directory dir = FSDirectory.open(new File("C:/IndexLast")); 
            Directory dir = FSDirectory.open(new File("C:/Index"));
            is = new IndexSearcher(dir);   //3 
            highlighter = new Highlighter(null);
            //docidMapper = mapper.getDocIDMapper("C:/IndexLast");
            docidMapper = mapper.getDocIDMapper("C:/Index");
            l = context.getLocation();
            user = context.getProfile();
        } catch (Exception ex) {
            System.err.printf("Cant construct context searcher");
        }
    }

    public String search(String q, int context, int cluster)
            throws Exception {
        //start to write log file
//        ql = new QueryLog(q, user.getID());
//        ql.writeLog();

        //query construct 
        Query query = parser.parse(vietString(q));              //4 
        Query queryhl = parser2.parse(vietString(q));
        System.out.println(q+context+cluster);
        //interest query
//        String qstr2 = interestStringConstruct();
//        System.out.println("SEARCHING:Interest string:" + qstr2);
//        Query query2 = parser.parse(qstr2);
//        Query queryhl2 = parser2.parse(qstr2);
//        Query[] qlist = {query, query2};
//        Query[] qlist2 = {queryhl, queryhl2};
//        query2 = query2.combine(qlist);
//        queryhl2 = queryhl2.combine(qlist2);
//        System.out.println("Q1: " + query.toString());
//        System.out.println("Q2: " + query2.toString());
        //searching
        TopDocs hits = normalSearch(is, query); //5
        if (context == 0) {
            return highlight(hits, query, queryhl);
        }
//        TopDocs hits2 = interestSearch(is, query2);
//        TopDocs hits3 = spatialSearch(is, query, l);
//        TopDocs hits4 = globalList(is, hits, hits2, hits3);
        TopDocs hits4 = globalList(is, hits);
        //eliminate
//            hits = eliminate(hits);
//            hits2 = eliminate(hits2);                       
        //show result
        //String hlString1 = highlight(hits,query,queryhl);
        //String hlString2 =highlight(hits2,query2,queryhl2);
        //String hlString3 =highlight(hits3,query,queryhl);
        if (cluster == 1) {
            return clusterResult(hits4, query, queryhl);
        }
        String hlString4 = highlight(hits4, query, queryhl);
        return hlString4;
    }

    public String interestStringConstruct() throws Exception {
        StringBuilder qbuf = new StringBuilder();
        String[] interestList = user.getInterest();
        for (int i = 0; i < interestList.length; i++) {
            if (interestList[i] != null) {
                qbuf.append(interestList[i]).append(" ");
            }
        }
        ArrayList<TaggedWord> arr = viA.reusableTokenStream(new StringReader(qbuf.toString())).getList();
        StringBuilder qbuf2 = new StringBuilder();
        for (int i = 0; i < arr.size(); i++) {
            qbuf2.append("\"").append(arr.get(i)).append("\"" + " ");
        }
        return qbuf2.toString();
    }

    public TopDocs normalSearch(IndexSearcher is, Query q) {
        try {
            TopDocs hits = is.search(q, 100); //5 
            return hits;
        } catch (Exception e) {//Catch exception if any
            System.err.println("Error: " + e.getMessage());
        }
        return null;
    }

//    public TopDocs interestSearch(IndexSearcher is, Query q) {
//        try {
//            TopDocs hits2 = is.search(q, 100);
//            return hits2;
//        } catch (Exception e) {//Catch exception if any
//            System.err.println("Error: " + e.getMessage());
//        }
//        return null;
//    }

//    public TopDocs spatialSearch(IndexSearcher is, Query q, Location l) throws IOException {
//        ManagerRecord mr = new ManagerRecord();
//        System.out.println("Begin spatial search");
//        LocationRecord lr;
//        ScoreDoc sd;
//        List<LocationRecord> resultList = mr.getKNearest(l.longitude, l.latitude, K);
//        ScoreDoc[] scoreList = new ScoreDoc[K];
//        System.out.println("Spatial result: " + resultList.size());
//        for (int i = 0; i < resultList.size(); i++) {
//            lr = resultList.get(i);
//            Explanation explaination = is.explain(q, docidMapper.getDocID(lr.getDocID()));
//            //float score_loc = (float) (500/(lr.distance(l.latitude,l.longitude)));
//            float score_loc = (float) (500 / lr.distance(l.latitude, l.longitude));
//            float score_doc = explaination.getValue();
//            sd = new ScoreDoc(docidMapper.getDocID(lr.getDocID()), score_loc + score_doc);
//            //System.out.println("i: "+docidMapper.getDocID(lr.getDocID())+">LOC "+score_loc+"-DOC "+score_doc);
//            scoreList[i] = sd;
//        }
//        TopDocs topDocs = new TopDocs(K, scoreList, 1);
//
//        return topDocs;
//    }

    public TopDocs globalList(IndexSearcher is, TopDocs top1) throws IOException {
        top1 = eliminate(top1);
        System.out.println("Top1 after eliminate: " + top1.scoreDocs.length);
//        top2 = eliminate(top2);
//        System.out.println("Top2 after eliminate: " + top2.scoreDocs.length);
        Map<Integer, Float> totalScore = new HashMap<>();
        ValComparator bvc = new ValComparator(totalScore);
        TreeMap<Integer, Float> sorted_map = new TreeMap(bvc);
        ScoreDoc[] list1 = top1.scoreDocs;
//        ScoreDoc[] list2 = top2.scoreDocs;
//        ScoreDoc[] list3 = top3.scoreDocs;
        //add query term to map

        for (int i = 0; i < list1.length; i++) {
            Float score = totalScore.get(list1[i].doc);
            totalScore.put(list1[i].doc, score == null ? list1[i].score : score + list1[i].score);
        }
//        for (int i = 0; i < list2.length; i++) {
//            Float score = totalScore.get(list2[i].doc);
//            totalScore.put(list2[i].doc, score == null ? list2[i].score * k_inte : score + list2[i].score * k_inte);
//        }
//        for (int i = 0; i < list3.length; i++) {
//            Float score = totalScore.get(list3[i].doc);
//            totalScore.put(list3[i].doc, score == null ? list3[i].score : score + list3[i].score);
//        }
        sorted_map.putAll(totalScore);
        int i = 0;
        ScoreDoc[] scoreDoc4 = new ScoreDoc[sorted_map.size()];
        //System.out.println(totalScore.size()+"-"+sorted_map.size());
        //System.out.println("unsorted map - nolimit");
//        for (Integer key : totalScore.keySet()) {
//            //System.out.println("key/value: " + key + "/"+totalScore.get(key));
//        }
        //System.out.println("results - nolimit");
        Iterator iterator = sorted_map.entrySet().iterator();
        while (iterator.hasNext()) {
            Map.Entry entry = (Map.Entry) iterator.next();
            //System.out.println("key/value: " + entry.getKey() + "/"+entry.getValue());
            ScoreDoc sd = new ScoreDoc((Integer) entry.getKey(), (Float) entry.getValue());
            scoreDoc4[i] = sd;
            i++;
        }

        return new TopDocs(scoreDoc4.length, scoreDoc4, scoreDoc4[0].score);
    }

    public TopDocs globalListLimit(IndexSearcher is, TopDocs top1, TopDocs top2, TopDocs top3) throws IOException {
        top1 = eliminate(top1);
        System.out.println("Top1: " + top1.scoreDocs.length);
        top2 = eliminate(top2);
        System.out.println("Top2: " + top2.scoreDocs.length);
        Map<Integer, Float> totalScore = new HashMap<>();
        ValComparator bvc = new ValComparator(totalScore);
        TreeMap<Integer, Float> sorted_map = new TreeMap(bvc);
        ScoreDoc[] list1 = top1.scoreDocs;
        ScoreDoc[] list2 = top2.scoreDocs;
        ScoreDoc[] list3 = top3.scoreDocs;
        int i = 0, j = 0, k = 0, tail = 0;
        ScoreDoc sd_tempt;
        do {
            if (list1[i].score >= list2[j].score && list1[i].score >= list3[k].score) {
                sd_tempt = list1[i];
                i++;
            } else if (list2[j].score >= list3[k].score) {
                sd_tempt = list2[j];
                j++;
            } else {
                sd_tempt = list3[k];
                k++;
            }
            Float score = totalScore.get(sd_tempt.doc);
            if (score == null) {
                tail++;
            }
            totalScore.put(sd_tempt.doc, score == null ? sd_tempt.score : score + sd_tempt.score);
        } while (tail < RESULT_LIMIT && i < list1.length);
        sorted_map.putAll(totalScore);
        int lt = 0;
        ScoreDoc[] scoreDoc4 = new ScoreDoc[sorted_map.size()];
//       System.out.println(totalScore.size()+"-"+sorted_map.size());
//       System.out.println("unsorted map - limit");
//        for (Integer key : totalScore.keySet()) {
//            System.out.println("key/value: " + key + "/"+totalScore.get(key));
//        }
//        System.out.println("results - limit");
        Iterator iterator = sorted_map.entrySet().iterator();
        while (iterator.hasNext()) {
            Map.Entry entry = (Map.Entry) iterator.next();
//            System.out.println("key/value: " + entry.getKey() + "/"+entry.getValue());
            ScoreDoc sd = new ScoreDoc((Integer) entry.getKey(), (Float) entry.getValue());
            scoreDoc4[lt] = sd;
            lt++;
        }

        return new TopDocs(scoreDoc4.length, scoreDoc4, scoreDoc4[0].score);
    }

    public TopDocs eliminate(TopDocs td) {
        ScoreDoc[] sdList = td.scoreDocs;
        int totalHit = 0;
        for (int i = 0; i < sdList.length; i++) {
            if (sdList[i].score < E) {
                break;

            }
            totalHit = i;
        }
        ScoreDoc[] sdList2 = new ScoreDoc[totalHit];
        System.arraycopy(sdList, 0, sdList2, 0, totalHit);
        td = new TopDocs(totalHit, sdList2, 1);
        System.out.println("Before eliminate: " + sdList.length);
        System.out.println("After eliminate: " + sdList2.length);
        return td;
    }

    public String vietString(String q) throws Exception {
        ArrayList<TaggedWord> arr = viA.reusableTokenStream(new StringReader(q)).getList();
        StringBuffer qbuf = new StringBuffer();
        for (int i = 0; i < arr.size(); i++) {
            qbuf.append("\"").append(arr.get(i)).append("\"" + " ");
        }
        return qbuf.toString();
    }

    public String highlight(TopDocs hits, Query q, Query qhl) throws Exception {
        StringBuffer buf = new StringBuffer();
        int limit = 50;
        if (hits.scoreDocs.length < 50) {
            limit = hits.scoreDocs.length;
        }
        for (int i = 0; i < limit; i++) {
            ScoreDoc scoreDoc = hits.scoreDocs[i];
            Document doc = is.doc(scoreDoc.doc);

            byte[] buffer = new byte[(int) new File(doc.get("filename")).length()];
            FileInputStream f = new FileInputStream(doc.get("filename"));
            f.read(buffer);
            TokenStream stream = new SimpleAnalyzer().tokenStream("field",
                    new StringReader(new String(buffer, "UTF-8")));
            QueryScorer scorer = new QueryScorer(qhl);
            Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
            highlighter.setFragmentScorer(scorer);
            highlighter.setTextFragmenter(fragmenter);
            String fragment =
                    highlighter.getBestFragment(stream, new String(buffer, "UTF-8"));
            if (fragment != null) {
                buf.append("<b>").append("<a href='content?link=").append(doc.get("filename")).append("'>").
                        append(doc.get("filename")).append("</a></b><br>");
//                buf.append("<b>").append("<a href='content.html'>").
//                        append(doc.get("filename")).append("</a></b><br>");
                buf.append(doc.get("filename")).append("<br>");
                Explanation explaination = is.explain(q, scoreDoc.doc);
                buf.append("  Score:").append(explaination.getValue()).append("  Score2:").append(scoreDoc.score).append("<br>");
                buf.append("...").append(fragment).append("...<br><br>");
            }

        }
        return buf.toString();
    }

    public String readFile(TopDocs hits) throws IOException {
        StringBuilder buf = new StringBuilder();
        ScoreDoc scoreDoc = hits.scoreDocs[1];
        Document doc = is.doc(scoreDoc.doc);
        FileInputStream f = new FileInputStream(doc.get("filename"));

        BufferedReader br = new BufferedReader(new InputStreamReader(f, "UTF-8"));
        String strLine;
        while ((strLine = br.readLine()) != null) {
            buf.append(strLine);
            System.out.println(">>>>>>>>>>>>>>CONTENT" + strLine);
        }
        BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
                new FileOutputStream("D://test1.txt"), "UTF-8"));
        try {
            out.write(buf.toString());
        } finally {
            out.close();
        }
        return buf.toString();
    }

    public String simpleHighlight(IndexSearcher is, Highlighter highlighter, TopDocs hits, Query q, Query qhl) throws Exception {
        StringBuilder buf = new StringBuilder();
        for (int i = 0; i < hits.scoreDocs.length; i++) {
            ScoreDoc scoreDoc = hits.scoreDocs[i];
            Document doc = is.doc(scoreDoc.doc);
            buf.append(i + 1).append(", File id: ").append(scoreDoc.doc);
            buf.append(", File path: ").append("<a href='").append(doc.get("filename")).append("'>").append("  Score:").append(scoreDoc.score).append("<br>");
        }
        return buf.toString();
    }

    public String clusterResult(TopDocs td, Query q, Query qhl) {
        try {
            ScoreDoc[] sd = td.scoreDocs;
            String[] arr = user.getInterest();
            TopDocs score_top1 = clusterInterestKey(arr[0], is, td);
            for (int i = 0; i < score_top1.scoreDocs.length; i++) {
                System.out.println(score_top1.scoreDocs[i]);
            }
            TopDocs score_top2 = clusterInterestKey(arr[1], is, td);
            TopDocs score_top3 = clusterInterestKey(arr[2], is, td);
            TopDocs score_top4 = clusterInterestKey(arr[3], is, td);
            TopDocs score_top5 = clusterInterestKey(arr[4], is, td);
            System.out.println("clustering results..............");
            StringBuilder buf = new StringBuilder();
            buf.append("<b>" + arr[0] + "</b><br>");
            buf.append(highlight(score_top1, q, qhl));
            buf.append("<br><hr>");
            buf.append("<b>").append(arr[1]).append("</b><br>");
            buf.append(highlight(score_top2, q, qhl));
            buf.append("<br><hr>");
            buf.append("<b>").append(arr[2]).append("</b><br>");
            buf.append(highlight(score_top3, q, qhl));
            buf.append("<br><hr>");
            buf.append("<b>").append(arr[3]).append("</b><br>");
            buf.append(highlight(score_top4, q, qhl));
            buf.append("<br><hr>");
            buf.append("<b>").append(arr[4]).append("</b><br>");
            buf.append(highlight(score_top5, q, qhl));
            buf.append("<br><hr>");
            return buf.toString();
        } catch (Exception ex) {
            ex.printStackTrace();
            System.err.println("cluster IO error!");
            return "Không thể phân cụm";
        }
    }

    public TopDocs clusterInterestKey(String key, IndexSearcher is, TopDocs td) {
        try {
            Map<Integer, Float> keyScores = new HashMap<>();
            ValComparator bvc = new ValComparator(keyScores);
            TreeMap<Integer, Float> sorted_map = new TreeMap(bvc);
            StringBuilder buf;
            //add query term to map
            for (int i = 0; i < td.scoreDocs.length; i++) {

                Explanation ex = is.explain(parser.parse(vietString(key)), td.scoreDocs[i].doc);
                keyScores.put(td.scoreDocs[i].doc, ex.getValue());
            }
            sorted_map.putAll(keyScores);
            int i = 0;
            ScoreDoc[] scoreDoc = new ScoreDoc[10];
            Iterator iterator = sorted_map.entrySet().iterator();
            while (iterator.hasNext() && i < 10) {
                Map.Entry entry = (Map.Entry) iterator.next();
                System.out.println("key/value: " + entry.getKey() + "/" + entry.getValue());
                ScoreDoc sd = new ScoreDoc((Integer) entry.getKey(), (Float) entry.getValue());
                scoreDoc[i] = sd;
                i++;
            }
            return new TopDocs(10, scoreDoc, scoreDoc[0].score);
        } catch (Exception ex1) {
            return null;
        }
    }

    class ValueComparator implements Comparator {

        Map base;

        public ValueComparator(Map base) {
            this.base = base;
        }

        @Override
        public int compare(Object a, Object b) {

            if ((Integer) base.get(a) < (Integer) base.get(b)) {
                return 1;
            } else if ((Integer) base.get(a) == (Integer) base.get(b)) {
                return -1;
            } else {
                return -1;
            }
        }
    }

    private class SDCompare implements Comparator<ScoreDoc> {

        @Override
        public int compare(ScoreDoc sd1, ScoreDoc sd2) {
            if (sd1.score > sd2.score) {
                return 1;
            }
            if (sd1.score == sd2.score) {
                return 0;
            }
            return -1;
        }
    }

    class ValComparator implements Comparator {

        Map base;

        public ValComparator(Map base) {
            this.base = base;
        }

        @Override
        public int compare(Object a, Object b) {

            if ((Float) base.get(a) < (Float) base.get(b)) {
                return 1;
            } else if ((Float) base.get(a) == (Float) base.get(b)) {
                return -1;
            } else {
                return -1;
            }
        }
    }
}
